/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */

package stanford.tagger;

import edu.stanford.nlp.tagger.maxent.TaggerConfig;
import edu.stanford.nlp.util.Pair;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import maxent.mr.io.FeatureReader;
import maxent.mr.trainer.FeatureRunner;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ToolRunner;


/**
 *  this class is modified versions of TaggerExperiments by Tautanova and Manning
 *  the differences, this class is using StanfordTagger instead of MaxentTagger class
 * @author hadoop
 */
public class TrainerExperiments extends Experiments implements Serializable{

    private static final boolean DEBUG = true;
    private static final String zeroSt = "0";

    static boolean isPopulated(int fNo, int size, StanfordTagger maxentTagger) {
        // Feature number 0 is hard-coded as the current word feature, which has a special threshold
        if (fNo == 0) {
          return (size > maxentTagger.curWordMinFeatureThresh);
        } else if (fNo < maxentTagger.extractors.getSize()) {
          return (size > maxentTagger.minFeatureThresh);
        } else {
          return (size > maxentTagger.rareWordMinFeatureThresh);
        }
    }

    private final TaggerFeatures feats;
    private Set<FeatureKey> sTemplates = new HashSet<FeatureKey>();
    private final HistoryTable tHistories = new HistoryTable();

    private final int numFeatsGeneral;
    private final int numFeatsAll;

    private final StanfordTagger maxentTagger;

    private final TemplateHash tFeature;
    private ReadDataTagged c;


    public TrainerExperiments(StanfordTagger tagger) {
        this.maxentTagger = tagger;
        this.tFeature = new TemplateHash(tagger);
        numFeatsGeneral = tagger.extractors.getSize();
        numFeatsAll = numFeatsGeneral + tagger.extractorsRare.getSize();
        feats = new TaggerFeatures(tagger.tags, this);
        
    }

    public TrainerExperiments(TaggerConfig config, StanfordTagger maxentTagger) throws IOException {
        this(maxentTagger);

        System.err.println("TaggerExperiments: adding word/tags");
        PairsHolder pairs = new PairsHolder();
        c = new ReadDataTagged(config, maxentTagger, pairs);
        vArray = new int[c.getSize()][2];
        System.err.println(c.getSize());
        initTemplatesNew();
        System.err.println("Featurizing tagged data tokens...");        


        // running MapReduce job for generating template
        // input and output path
        String[] args = new String[]{"/user/hadoop/rdt","/user/hadoop/fk-out"};                
        try {
            ToolRunner.run(new Configuration(), new FeatureRunner(this), args);
        } catch (Exception ex) {
            ex.printStackTrace();
        }

        FeatureReader fReader = new FeatureReader(args[1]);
        fReader.read();
        sTemplates = fReader.getsTemplates();

        for (int i = 0, size = c.getSize(); i < size; i++) {

            DataWordTag d = c.get(i);
            String yS = d.getY();
            History h = d.getHistory();
            int indX = tHistories.add(h);
            int indY = d.getYInd();
//            addTemplatesNew(h, yS);
//            addRareTemplatesNew(h, yS);
            vArray[i][0] = indX;
            vArray[i][1] = indY;

            if (i > 0 && (i % 10000) == 0) {
                System.err.printf("%d ",i);
                if (i % 100000 == 0) { System.err.println(); }
            }
            
        }
        System.err.println();
        System.err.println("Featurized " + c.getSize() + " data tokens [done].");
        c.release();
        ptilde();
        maxentTagger.xSize = xSize;
        maxentTagger.ySize = ySize;
        System.err.println("xSize [num Phi templates] = " + xSize + "; ySize [num classes] = " + ySize);

        hashHistories();

        // if we'll look at occurring tags only, we need the histories and pairs still
        if (!maxentTagger.occuringTagsOnly && !maxentTagger.possibleTagsOnly) {
        tHistories.release();
        pairs.clear();
        }

        getFeaturesNew();
    }

    private void initTemplatesNew() {
        maxentTagger.dict.setAmbClasses(maxentTagger.ambClasses, maxentTagger.veryCommonWordThresh, maxentTagger.tags);
    }

    // Add a new feature key in a hashtable of feature templates
    private void addTemplatesNew(History h, String tag) {
        // Feature templates general

        for (int i = 0; i < numFeatsGeneral; i++) {
          String s = maxentTagger.extractors.extract(i, h);
          if (s.equals(zeroSt)) {
            continue;
          } //do not add the feature
          //iterate over tags in dictionary
          if (maxentTagger.alltags) {
            int numTags = maxentTagger.tags.getSize();
            for (int j = 0; j < numTags; j++) {

              String tag1 = maxentTagger.tags.getTag(j);

              FeatureKey key = new FeatureKey(i, s, tag1);

              if (!maxentTagger.extractors.get(i).precondition(tag1)) {
                continue;
              }

              add(key);
            }
          } else {
            //only this tag
            FeatureKey key = new FeatureKey(i, s, tag);

            if (!maxentTagger.extractors.get(i).precondition(tag)) {
              continue;
            }

            add(key);
          }
        }
    }

    /** Adds a FeatureKey to the set of known FeatureKeys.
   *
   * @param s The feature key to be added
   * @return Whether the key was already known (false) or added (true)
   */
  protected boolean add(FeatureKey s) {
    if ((sTemplates.contains(s))) {
      return false;
    }
    sTemplates.add(s);
    return true;
  }

    private void addRareTemplatesNew(History h, String tag) {
        // Feature templates rare

        if (!(maxentTagger.isRare(ExtractorFramesGeneral.cWord.extract(h)))) {
          return;
        }
        int start = numFeatsGeneral;
        for (int i = start; i < numFeatsAll; i++) {
          String s = maxentTagger.extractorsRare.extract(i - start, h);

          if (s.equals(zeroSt)) {
            continue;
          } //do not add the feature
          if (maxentTagger.alltags) {
            int numTags = maxentTagger.tags.getSize();
            for (int j = 0; j < numTags; j++) {

              String tag1 = maxentTagger.tags.getTag(j);

              FeatureKey key = new FeatureKey(i, s, tag1);

              if (!maxentTagger.extractorsRare.get(i - start).precondition(tag1)) {
                continue;
              }

              add(key);
            }
          } else {
            //only this tag
            FeatureKey key = new FeatureKey(i, s, tag);

            if (!maxentTagger.extractorsRare.get(i - start).precondition(tag)) {
              continue;
            }

            add(key);
          }
        }
    }

    private void hashHistories() {
        int fAll = maxentTagger.extractors.getSize() + maxentTagger.extractorsRare.getSize();
        int fGeneral = maxentTagger.extractors.getSize();
        System.err.println("Hashing histories ...");
        for (int x = 0; x < xSize; x++) {
          History h = tHistories.getHistory(x);
          if (x > 0 && x % 10000 == 0) {
            System.err.printf("%d ",x);
            if (x % 100000 == 0) { System.err.println(); }
          }
          int fSize = (maxentTagger.isRare(ExtractorFramesGeneral.cWord.extract(h)) ? fAll : fGeneral);
          for (int i = 0; i < fSize; i++) {
            tFeature.addPrev(i, h);
          }
        } // for x
        // now for the populated ones
        System.err.println();
        System.err.println("Hashed " + xSize + " histories.");
        System.err.println("Hashing populated histories ...");
        for (int x = 0; x < xSize; x++) {
          History h = tHistories.getHistory(x);
          if (x > 0 && x % 10000 == 0) {
            System.err.print(x + " ");
            if (x % 100000 == 0) { System.err.println(); }
          }
          int fSize = (maxentTagger.isRare(ExtractorFramesGeneral.cWord.extract(h)) ? fAll : fGeneral);
          for (int i = 0; i < fSize; i++) {
            tFeature.add(i, h, x); // write this to check whether to add
          }
        } // for x
        System.err.println();
        System.err.println("Hashed populated histories.");
    }

    private void getFeaturesNew() {
        try {
          System.out.println("TaggerExperiments.getFeaturesNew: initializing fnumArr.");
          maxentTagger.fnumArr = new byte[xSize][ySize]; // what is the maximum number of active features
          File hFile = File.createTempFile("temp",".x", new File("./"));
          RandomAccessFile hF = new RandomAccessFile(hFile, "rw");
          System.out.println("  length of sTemplates keys: " + sTemplates.size());
          System.out.println("getFeaturesNew adding features ...");
          int current = 0;
          int numFeats = 0;
          final boolean VERBOSE = false;
          for (FeatureKey fK : sTemplates) {
            int numF = fK.num;
            int[] xValues;
            Pair<Integer, String> wT = new Pair<Integer, String>(numF, fK.val);
            xValues = tFeature.getXValues(wT);
            if (xValues == null) {
              System.out.println("  xValues is null: " + fK.toString()); //  + " " + i
              continue;
            }
            int numEvidence = 0;
            int y = maxentTagger.tags.getIndex(fK.tag);
            for (int xValue : xValues) {

              if (maxentTagger.occuringTagsOnly) {
                //check whether the current word in x has occurred with y
                String word = ExtractorFramesGeneral.cWord.extract(tHistories.getHistory(xValue));
                if (maxentTagger.dict.getCount(word, fK.tag) == 0) {
                  continue;
                }
              }
              if (maxentTagger.possibleTagsOnly) {
                String word = ExtractorFramesGeneral.cWord.extract(tHistories.getHistory(xValue));
                String[] tags = maxentTagger.dict.getTags(word);
                Set<String> s = new HashSet<String>(Arrays.asList(maxentTagger.tags.deterministicallyExpandTags(tags, word)));
                if(DEBUG)
                  System.err.printf("possible tags for %s: %s\n", word, Arrays.toString(s.toArray()));
                if(!s.contains(fK.tag))
                  continue;
              }
              numEvidence += this.px[xValue];
            }

            if (populated(numF, numEvidence)) {
              int[] positions = tFeature.getPositions(fK);
              if (maxentTagger.occuringTagsOnly || maxentTagger.possibleTagsOnly) { // TODO
                positions = null;
              }

              if (positions == null) {
                // write this in the file and create a TaggerFeature for it
                //int numElem
                int numElements = 0;

                for (int x : xValues) {
                  if (maxentTagger.occuringTagsOnly) {
                    //check whether the current word in x has occurred with y
                    String word = ExtractorFramesGeneral.cWord.extract(tHistories.getHistory(x));
                    if (maxentTagger.dict.getCount(word, fK.tag) == 0) {
                      continue;
                    }
                  }
                  if(maxentTagger.possibleTagsOnly) {
                    String word = ExtractorFramesGeneral.cWord.extract(tHistories.getHistory(x));
                    String[] tags = maxentTagger.dict.getTags(word);
                    Set<String> s = new HashSet<String>(Arrays.asList(maxentTagger.tags.deterministicallyExpandTags(tags, word)));
                    if(!s.contains(fK.tag))
                      continue;
                  }
                  numElements++;

                  hF.writeInt(x);
                  maxentTagger.fnumArr[x][y]++;
                }
                TaggerFeature tF = new TaggerFeature(current, current + numElements - 1, fK,
                                                     maxentTagger.tags, this);
                tFeature.addPositions(current, current + numElements - 1, fK);
                current = current + numElements;
                feats.add(tF);
               // if (VERBOSE) {
                  System.out.println("  added feature with key " + fK.toString() + " has support " + numElements);
                //}
              } else {

                for(int x : xValues) {
                  maxentTagger.fnumArr[x][y]++;
                }
                // this is the second time to write these values
                TaggerFeature tF = new TaggerFeature(positions[0], positions[1], fK,
                                                     maxentTagger.tags, this);
                feats.add(tF);
                if (VERBOSE) {
                  System.out.println("  added feature with key " + fK.toString() + " has support " + xValues.length);
                }
              }

              maxentTagger.fAssociations.put(fK, numFeats);
              numFeats++;
            }

          } // foreach FeatureKey fK
          // read out the file and put everything in an array of ints stored in Feats
          tFeature.release();
          feats.xIndexed = new int[current];
          hF.seek(0);
          int current1 = 0;
          while (current1 < current) {
            feats.xIndexed[current1] = hF.readInt();
            current1++;
          }
          System.out.println("  total feats: " + sTemplates.size() + ", populated: " + numFeats);
          hF.close();
          hFile.delete();

          // what is the maximum number of active features per pair
          int max = 0;
          int maxGt = 0;
          int numZeros = 0;
          for (int x = 0; x < xSize; x++) {
            int numGt = 0;
            for (int y = 0; y < ySize; y++) {
              if (maxentTagger.fnumArr[x][y] > 0) {
                numGt++;
                if (max < maxentTagger.fnumArr[x][y]) {
                  max = maxentTagger.fnumArr[x][y];
                }
              } else {
                // if 00
                numZeros++;
              }
            }
            if (maxGt < numGt) {
              maxGt = numGt;
            }
          } // for x

          System.out.println("  Max features per x,y pair: " + max);
          System.out.println("  Max non-zero y values for an x: " + maxGt);
          System.out.println("  Number of non-zero feature x,y pairs: " +
              (xSize * ySize - numZeros));
          System.out.println("  Number of zero feature x,y pairs: " + numZeros);
          System.out.println("end getFeaturesNew.");
        } catch (Exception e) {
          e.printStackTrace();
        }
    }

    private boolean populated(int fNo, int size) {
        return isPopulated(fNo, size, maxentTagger);
    }

    public TaggerFeatures getTaggerFeatures() {
        return feats;
    }


    // Add a new feature key in a hashtable of feature templates
    public FeatureKey writeTemplatesNew(History h, String tag) {
        // Feature templates general

        for (int i = 0; i < numFeatsGeneral; i++) {
          String s = maxentTagger.extractors.extract(i, h);

          if (s.equals(zeroSt)) {
            continue;
          } //do not add the feature

          //iterate over tags in dictionary
          if (maxentTagger.alltags) {
            int numTags = maxentTagger.tags.getSize();
            for (int j = 0; j < numTags; j++) {

              String tag1 = maxentTagger.tags.getTag(j);

              FeatureKey key = new FeatureKey(i, s, tag1);

              if (!maxentTagger.extractors.get(i).precondition(tag1)) {
                continue;
              }

              return key;
            }
          } else {
            //only this tag
            FeatureKey key = new FeatureKey(i, s, tag);

            if (!maxentTagger.extractors.get(i).precondition(tag)) {
              continue;
            }

            return key;
          }
        }
        return null;
    }

    public FeatureKey writeRareTemplatesNew(History h, String tag) {
        // Feature templates rare

        if (!(maxentTagger.isRare(ExtractorFramesGeneral.cWord.extract(h)))) {
          return null;
        }
        int start = numFeatsGeneral;
        for (int i = start; i < numFeatsAll; i++) {
          String s = maxentTagger.extractorsRare.extract(i - start, h);

          if (s.equals(zeroSt)) {
            continue;
          } //do not add the feature
          if (maxentTagger.alltags) {
            int numTags = maxentTagger.tags.getSize();
            for (int j = 0; j < numTags; j++) {

              String tag1 = maxentTagger.tags.getTag(j);

              FeatureKey key = new FeatureKey(i, s, tag1);

              if (!maxentTagger.extractorsRare.get(i - start).precondition(tag1)) {
                continue;
              }

              return key;
            }
          } else {
            //only this tag
            FeatureKey key = new FeatureKey(i, s, tag);

            if (!maxentTagger.extractorsRare.get(i - start).precondition(tag)) {
              continue;
            }

            return key;
          }
        }
        return null;
    }

    public ReadDataTagged getReadDataTagged() {
        return c;
    }

    public StanfordTagger getMaxentTagger() {
        return maxentTagger;
    }

    public int getNumFeatsAll() {
        return numFeatsAll;
    }

    public int getNumFeatsGeneral() {
        return numFeatsGeneral;
    }

    

}
